<!DOCTYPE html>
<html lang="zh">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.3.0">


  <link rel="apple-touch-icon" sizes="180x180" href="/yuwanzi.io/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/yuwanzi.io/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/yuwanzi.io/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/yuwanzi.io/images/logo.svg" color="#222">

<link rel="stylesheet" href="/yuwanzi.io/css/main.css">



<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.15.1/css/all.min.css">
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/animate.css@3.1.1/animate.min.css">

<script class="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"suyuhuan.gitee.io","root":"/yuwanzi.io/","images":"/yuwanzi.io/images","scheme":"Muse","version":"8.2.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12},"copycode":false,"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"fadeInDown","post_body":"fadeInDown","coll_header":"fadeInLeft","sidebar":"fadeInUp"}},"prism":false,"i18n":{"placeholder":"Suche...","empty":"We didn't find any results for the search: ${query}","hits_time":"${hits} results found in ${time} ms","hits":"${hits} results found"}};
  </script>
<meta name="description" content="概述&amp;nbsp;&amp;nbsp;HA(High Available), 高可用性集群，是保证业务连续性的有效解决方案，一般有两个或两个以上的节点，且分为活动节点及备用节点。通常把正在执行业务的称为活动节点，而作为活动节点的一个备份的则称为备用节点。当活动节点出现问题，导致正在运行的业务（任务）不能正常运行时，备用节点此时就会侦测到，并立即接续活动节点来执行业务。从而实现业务的不中断或短暂中断。 &amp;n">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop学习笔记(3)-HA高可用集群搭建">
<meta property="og:url" content="https://suyuhuan.gitee.io/yuwanzi.io/2016/07/15/2016-07-15-Hadoop03-HA/index.html">
<meta property="og:site_name" content="玉丸子 | Blog">
<meta property="og:description" content="概述&amp;nbsp;&amp;nbsp;HA(High Available), 高可用性集群，是保证业务连续性的有效解决方案，一般有两个或两个以上的节点，且分为活动节点及备用节点。通常把正在执行业务的称为活动节点，而作为活动节点的一个备份的则称为备用节点。当活动节点出现问题，导致正在运行的业务（任务）不能正常运行时，备用节点此时就会侦测到，并立即接续活动节点来执行业务。从而实现业务的不中断或短暂中断。 &amp;n">
<meta property="og:locale">
<meta property="og:image" content="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w446w8pcj20bp08rweu.jpg">
<meta property="og:image" content="http://ww2.sinaimg.cn/mw690/63503acbjw1f5w61d75ixj20p40jidhc.jpg">
<meta property="article:published_time" content="2016-07-15T10:00:00.000Z">
<meta property="article:modified_time" content="2020-11-07T00:58:17.000Z">
<meta property="article:author" content="玉丸子">
<meta property="article:tag" content="Hadoop">
<meta property="article:tag" content="大数据">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w446w8pcj20bp08rweu.jpg">


<link rel="canonical" href="https://suyuhuan.gitee.io/yuwanzi.io/2016/07/15/2016-07-15-Hadoop03-HA/">


<script class="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh'
  };
</script>
<title>Hadoop学习笔记(3)-HA高可用集群搭建 | 玉丸子 | Blog</title>
  




  <noscript>
  <style>
  body { margin-top: 2rem; }

  .use-motion .menu-item,
  .use-motion .sidebar,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header {
    visibility: visible;
  }

  .use-motion .header,
  .use-motion .site-brand-container .toggle,
  .use-motion .footer { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle,
  .use-motion .custom-logo-image {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line {
    transform: scaleX(1);
  }

  .search-pop-overlay, .sidebar-nav { display: none; }
  .sidebar-panel { display: block; }
  </style>
</noscript>

<link rel="alternate" href="/yuwanzi.io/atom.xml" title="玉丸子 | Blog" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage" class="use-motion">
  <div class="headband"></div>

  <main class="main">
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Navigationsleiste an/ausschalten" role="button">
    </div>
  </div>

  <div class="site-meta">

    <a href="/yuwanzi.io/" class="brand" rel="start">
      <i class="logo-line"></i>
      <h1 class="site-title">玉丸子 | Blog</h1>
      <i class="logo-line"></i>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>







</div>
        
  
  <div class="toggle sidebar-toggle" role="button">
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
    <span class="toggle-line"></span>
  </div>

  <aside class="sidebar">

    <div class="sidebar-inner sidebar-nav-active sidebar-toc-active">
      <ul class="sidebar-nav">
        <li class="sidebar-nav-toc">
          Inhaltsverzeichnis
        </li>
        <li class="sidebar-nav-overview">
          Übersicht
        </li>
      </ul>

      <div class="sidebar-panel-container">
        <!--noindex-->
        <div class="post-toc-wrap sidebar-panel">
            <div class="post-toc animated"><ol class="nav"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E6%A6%82%E8%BF%B0"><span class="nav-number">1.</span> <span class="nav-text">概述</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HA%E6%9E%B6%E6%9E%84"><span class="nav-number">2.</span> <span class="nav-text">HA架构</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E6%90%AD%E5%BB%BAHA%E9%9B%86%E7%BE%A4"><span class="nav-number">3.</span> <span class="nav-text">搭建HA集群</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#%E6%B5%8B%E8%AF%95%E9%9B%86%E7%BE%A4%E8%A7%84%E5%88%92"><span class="nav-number">3.1.</span> <span class="nav-text">测试集群规划</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#ssh%E5%85%8D%E5%AF%86%E7%99%BB%E9%99%86"><span class="nav-number">3.2.</span> <span class="nav-text">ssh免密登陆</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#%E5%AE%89%E8%A3%85zookeeper"><span class="nav-number">3.3.</span> <span class="nav-text">安装zookeeper</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#core-site-xml"><span class="nav-number">3.4.</span> <span class="nav-text">core-site.xml</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#hdfs-site-xml"><span class="nav-number">3.5.</span> <span class="nav-text">hdfs-site.xml</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#mapred-site-xml"><span class="nav-number">3.6.</span> <span class="nav-text">mapred-site.xml</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#yarn-site-xml"><span class="nav-number">3.7.</span> <span class="nav-text">yarn-site.xml</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#slaves"><span class="nav-number">3.8.</span> <span class="nav-text">slaves</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%90%AF%E5%8A%A8HA%E9%9B%86%E7%BE%A4"><span class="nav-number">4.</span> <span class="nav-text">启动HA集群</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E7%AE%A1%E7%90%86%E5%91%BD%E4%BB%A4"><span class="nav-number">5.</span> <span class="nav-text">管理命令</span></a></li></ol></div>
        </div>
        <!--/noindex-->

        <div class="site-overview-wrap sidebar-panel">
          <div class="site-author site-overview-item animated" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">玉丸子</p>
  <div class="site-description" itemprop="description">这里是玉丸子的个人博客,与你一起发现更大的世界。</div>
</div>
<div class="site-state-wrap site-overview-item animated">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/yuwanzi.io/archives">
          <span class="site-state-item-count">68</span>
          <span class="site-state-item-name">Artikel</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/yuwanzi.io/categories/">
        <span class="site-state-item-count">39</span>
        <span class="site-state-item-name">Kategorien</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/yuwanzi.io/tags/">
        <span class="site-state-item-count">46</span>
        <span class="site-state-item-name">schlagwörter</span></a>
      </div>
  </nav>
</div>



        </div>
      </div>
    </div>
  </aside>
  <div class="sidebar-dimmer"></div>


    </header>

    
  <div class="back-to-top" role="button">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>

<noscript>
  <div class="noscript-warning">Theme NexT works best with JavaScript enabled</div>
</noscript>


    <div class="main-inner post posts-expand">


  


<div class="post-block">
  
  

  <article itemscope itemtype="http://schema.org/Article" class="post-content" lang="zh">
    <link itemprop="mainEntityOfPage" href="https://suyuhuan.gitee.io/yuwanzi.io/2016/07/15/2016-07-15-Hadoop03-HA/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/yuwanzi.io/images/avatar.gif">
      <meta itemprop="name" content="玉丸子">
      <meta itemprop="description" content="这里是玉丸子的个人博客,与你一起发现更大的世界。">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="玉丸子 | Blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Hadoop学习笔记(3)-HA高可用集群搭建
        </h1>

        <div class="post-meta-container">
          <div class="post-meta">
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-calendar"></i>
      </span>
      <span class="post-meta-item-text">Veröffentlicht am</span>

      <time title="Erstellt: 2016-07-15 18:00:00" itemprop="dateCreated datePublished" datetime="2016-07-15T18:00:00+08:00">2016-07-15</time>
    </span>
      <span class="post-meta-item">
        <span class="post-meta-item-icon">
          <i class="far fa-calendar-check"></i>
        </span>
        <span class="post-meta-item-text">Bearbeitet am</span>
        <time title="Geändert am: 2020-11-07 08:58:17" itemprop="dateModified" datetime="2020-11-07T08:58:17+08:00">2020-11-07</time>
      </span>
    <span class="post-meta-item">
      <span class="post-meta-item-icon">
        <i class="far fa-folder"></i>
      </span>
      <span class="post-meta-item-text">in</span>
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/" itemprop="url" rel="index"><span itemprop="name">后端</span></a>
        </span>
          . 
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/%E5%A4%A7%E6%95%B0%E6%8D%AE/" itemprop="url" rel="index"><span itemprop="name">大数据</span></a>
        </span>
          . 
        <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
          <a href="/yuwanzi.io/categories/%E5%90%8E%E7%AB%AF/%E5%A4%A7%E6%95%B0%E6%8D%AE/Hadoop/" itemprop="url" rel="index"><span itemprop="name">Hadoop</span></a>
        </span>
    </span>

  
</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">
        <p><img src="http://ww4.sinaimg.cn/mw690/63503acbjw1f5w446w8pcj20bp08rweu.jpg"></p>
<h3 id="概述"><a href="#概述" class="headerlink" title="概述"></a>概述</h3><p>&nbsp;&nbsp;HA(High Available), 高可用性集群，是保证业务连续性的有效解决方案，一般有两个或两个以上的节点，且分为活动节点及备用节点。通常把正在执行业务的称为活动节点，而作为活动节点的一个备份的则称为备用节点。当活动节点出现问题，导致正在运行的业务（任务）不能正常运行时，备用节点此时就会侦测到，并立即接续活动节点来执行业务。从而实现业务的不中断或短暂中断。</p>
<p>&nbsp;&nbsp;在hadoop2.0之前,每个集群只有一个NameNode,如果那台机器坏掉,集群作为一个整体将不可用,所以为了解决这个问题Hadoop2.0引入了HA机制,可以通过在同一集群上配置运行两个冗余的NameNodes，做到主动/被动的热备份。这将允许当一个机器宕机时，快速转移到一个新的NameNode，或管理员进行利用故障转移达到优雅的系统升级的目的。</p>
<p>&nbsp;&nbsp;HA一共有二种解决方案，一种是NFS（Network File System）方式，另外一种是QJM（Quorum Journal Manager）方式。</p>
<h3 id="HA架构"><a href="#HA架构" class="headerlink" title="HA架构"></a>HA架构</h3><p><img src="http://ww2.sinaimg.cn/mw690/63503acbjw1f5w61d75ixj20p40jidhc.jpg"></p>
<p>&nbsp;&nbsp;一个典型的HA集群，NameNode会被配置在两台独立的机器上.在任何的时间上，一个NameNode处于活动状态，而另一个在备份状态，活动状态的NameNode会响应集群中所有的客户端，同时备份的只是作为一个副本，保证在必要的时候提供一个快速的转移。</p>
<p>&nbsp;&nbsp;为了使备份的节点和活动的节点保持一致，两个节点通过一个特殊的守护线程相连，这个线程叫做“JournalNodes”（JNs）。当活动状态的节点修改任何的命名空间，他都会通过这些JNs记录日志，备用的节点可以监控edit日志的变化，并且通过JNs读取到变化。备份节点查看edits可以拥有专门的namespace。在故障转移的时候备份节点将在切换至活动状态前确认他从JNs读取到的所有edits。这个确认的目的是为了保证Namespace的状态和迁移之前是完全同步的。</p>
<p>&nbsp;&nbsp;为了提供一个快速的转移，备份NameNode要求保存着最新的block在集群当中的信息。为了能够得到这个，DataNode都被配置了所有的NameNode的地址，并且发送block的地址信息和心跳给两个node。</p>
<p>&nbsp;&nbsp;保证只有一个活跃的NameNode在集群当中是一个十分重要的一步。否则namespace状态在两个节点间不同会导致数据都是或者其他一些不正确的结果。为了确保这个,防止所谓split - brain场景,JournalNodes将只允许一个NameNode进行写操作。故障转移期间,NameNode成为活跃状态的时候会接管JournalNodes的写权限,这会有效防止其他NameNode持续处于活跃状态,允许新的活动节点安全进行故障转移。</p>
<h3 id="搭建HA集群"><a href="#搭建HA集群" class="headerlink" title="搭建HA集群"></a>搭建HA集群</h3><p>&nbsp;&nbsp;hadoop-2.2.0中依然存在一个问题，就是ResourceManager只有一个，存在单点故障，hadoop-2.4.1解决了这个问题，可以有两个ResourceManager，一个是Active，一个是Standby，状态由zookeeper进行协调。</p>
<h4 id="测试集群规划"><a href="#测试集群规划" class="headerlink" title="测试集群规划"></a>测试集群规划</h4><p>&nbsp;&nbsp;实验使用7台虚拟机,规划如下:</p>
<table>
<thead>
<tr>
<th>HostName</th>
<th>IP</th>
<th>Software</th>
<th>Process</th>
</tr>
</thead>
<tbody><tr>
<td>datanode01</td>
<td>192.168.145.140</td>
<td>jdk、hadoop、zookeeper</td>
<td>DataNode、NodeManager、JournalNode、QuorumPeerMain</td>
</tr>
<tr>
<td>datanode02</td>
<td>192.168.145.141</td>
<td>jdk、hadoop、zookeeper</td>
<td>DataNode、NodeManager、JournalNode、QuorumPeerMain</td>
</tr>
<tr>
<td>datanode03</td>
<td>192.168.145.142</td>
<td>jdk、hadoop、zookeeper</td>
<td>DataNode、NodeManager、JournalNode、QuorumPeerMain</td>
</tr>
<tr>
<td>namenode01</td>
<td>192.168.145.143</td>
<td>jdk、hadoop</td>
<td>NameNode、DFSZKFailoverController(ZKFC)</td>
</tr>
<tr>
<td>namenode02</td>
<td>192.168.145.144</td>
<td>jdk、hadoop</td>
<td>NameNode、DFSZKFailoverController(ZKFC)</td>
</tr>
<tr>
<td>yarn01</td>
<td>192.168.145.145</td>
<td>jdk、hadoop</td>
<td>ResourceManager</td>
</tr>
<tr>
<td>yarn02</td>
<td>192.168.145.146</td>
<td>jdk、hadoop</td>
<td>ResourceManager</td>
</tr>
</tbody></table>
<h4 id="ssh免密登陆"><a href="#ssh免密登陆" class="headerlink" title="ssh免密登陆"></a>ssh免密登陆</h4><ul>
<li>namenode01需要配置所有datanode、yarn、namenode的免密登陆。</li>
<li>namenode02需要配置namenode01的免密登陆。</li>
<li>yarn01需要配置所有nodemanager与resourcemanager的免密登陆。</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">#在namenode01上生成密匙</span><br><span class="line">ssh-keygen</span><br><span class="line">#namenode01拷贝密匙(包括自己)</span><br><span class="line">ssh-copy-id namenode01</span><br><span class="line">ssh-copy-id namenode02</span><br><span class="line">ssh-copy-id datanode01</span><br><span class="line">ssh-copy-id datanode02</span><br><span class="line">ssh-copy-id datanode03</span><br><span class="line">ssh-copy-id yarn01</span><br><span class="line">ssh-copy-id yarn02</span><br><span class="line"></span><br><span class="line">#在namenode02上生成密匙</span><br><span class="line">ssh-keygen</span><br><span class="line">ssh-copy-id namenode01</span><br><span class="line">ssh-copy-id namenode02</span><br><span class="line"></span><br><span class="line">#在yarn01上生成密匙</span><br><span class="line">ssh-keygen</span><br><span class="line">#yarn01拷贝密匙</span><br><span class="line">ssh-copy-id yarn02</span><br><span class="line">ssh-copy-id datanode01</span><br><span class="line">ssh-copy-id datanode02</span><br><span class="line">ssh-copy-id datanode03</span><br></pre></td></tr></table></figure>
<h4 id="安装zookeeper"><a href="#安装zookeeper" class="headerlink" title="安装zookeeper"></a>安装zookeeper</h4><ol>
<li>解压zookeeper</li>
<li>重命名zookeeper/conf下的zoo_sample.cfg为zoo.cfg : mv zoo_sample.cfg zoo.cfg</li>
<li>在zoo.cfg中修改dataDir=$ZOOKEEPERHOME/data 这个文件需要自己创建<br>例如:dataDir=/home/application/zookeeper-3.4.5/data</li>
<li>在zoo.cfg中最后添加 server.id=ip:2888:3888<br>例如:<pre><code>server.1=datanode01:2888:3888
server.2=datanode02:2888:3888
server.3=datanode03:2888:3888</code></pre>
</li>
<li>在$ZOOKEEPERHOME/data目录中创建一个myid文件并写入id。<br>例如: echo 1 &gt;/home/application/zookeeper-3.4.5/data/myid<br>id需要跟zoo.cfg中配置的一致。</li>
</ol>
<h4 id="core-site-xml"><a href="#core-site-xml" class="headerlink" title="core-site.xml"></a>core-site.xml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- 指定hdfs的nameservice为ns1 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;fs.defaultFS&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;hdfs:&#x2F;&#x2F;ns1&#x2F;&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 指定hadoop临时目录 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;hadoop.tmp.dir&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;&#x2F;home&#x2F;application&#x2F;hadoop-2.6.0&#x2F;tmp&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;				</span><br><span class="line">	&lt;!-- 指定zookeeper地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;ha.zookeeper.quorum&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;datanode01:2181,datanode02:2181,datanode03:2181&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">&lt;&#x2F;configuration&gt;</span><br></pre></td></tr></table></figure>
<h4 id="hdfs-site-xml"><a href="#hdfs-site-xml" class="headerlink" title="hdfs-site.xml"></a>hdfs-site.xml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!--指定hdfs的nameservice为ns1，需要和core-site.xml中的保持一致.</span><br><span class="line">	  这个名字是逻辑名字,可以是任意的,它将被用来配置在集群中作为HDFS的绝对路径组件。--&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.nameservices&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;ns1&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- ns1下面有两个NameNode，分别是nn1，nn2 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.ha.namenodes.ns1&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;nn1,nn2&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- nn1的RPC通信地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.namenode.rpc-address.ns1.nn1&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;namenode01:9000&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- nn1的http通信地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.namenode.http-address.ns1.nn1&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;namenode01:50070&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- nn2的RPC通信地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.namenode.rpc-address.ns1.nn2&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;namenode02:9000&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- nn2的http通信地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.namenode.http-address.ns1.nn2&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;namenode02:50070&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 指定NameNode的元数据在JournalNode上的存放位置 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.namenode.shared.edits.dir&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;qjournal:&#x2F;&#x2F;datanode01:8485;datanode02:8485;datanode03:8485&#x2F;ns1&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 指定JournalNode在本地磁盘存放数据的位置 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.journalnode.edits.dir&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;&#x2F;home&#x2F;application&#x2F;hadoop-2.6.0&#x2F;journaldata&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 开启NameNode失败自动切换 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.ha.automatic-failover.enabled&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;true&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 配置失败自动切换实现方式 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.client.failover.proxy.provider.ns1&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 配置隔离机制方法，多个机制用换行分割，即每个机制暂用一行--&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.ha.fencing.methods&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;</span><br><span class="line">			sshfence</span><br><span class="line">			shell(&#x2F;bin&#x2F;true)</span><br><span class="line">		&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 使用sshfence隔离机制时需要ssh免登陆 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.ha.fencing.ssh.private-key-files&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;&#x2F;root&#x2F;.ssh&#x2F;id_rsa&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 配置sshfence隔离机制超时时间 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;dfs.ha.fencing.ssh.connect-timeout&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;30000&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">&lt;&#x2F;configuration&gt;</span><br></pre></td></tr></table></figure>
<h4 id="mapred-site-xml"><a href="#mapred-site-xml" class="headerlink" title="mapred-site.xml"></a>mapred-site.xml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- 指定mr框架为yarn方式 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;mapreduce.framework.name&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;yarn&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">&lt;&#x2F;configuration&gt;</span><br></pre></td></tr></table></figure>
<h4 id="yarn-site-xml"><a href="#yarn-site-xml" class="headerlink" title="yarn-site.xml"></a>yarn-site.xml</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">	&lt;!-- 开启RM高可用 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.resourcemanager.ha.enabled&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;true&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 指定RM的cluster id 这是一个逻辑名称,可以是任意的 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.resourcemanager.cluster-id&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;yarncluster&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 指定RM的名字 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.resourcemanager.ha.rm-ids&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;rm1,rm2&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 分别指定RM的地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.resourcemanager.hostname.rm1&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;yarn01&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.resourcemanager.hostname.rm2&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;yarn02&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;!-- 指定zk集群地址 --&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.resourcemanager.zk-address&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;datanode01:2181,datanode02:2181,datanode03:2181&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">	&lt;property&gt;</span><br><span class="line">		&lt;name&gt;yarn.nodemanager.aux-services&lt;&#x2F;name&gt;</span><br><span class="line">		&lt;value&gt;mapreduce_shuffle&lt;&#x2F;value&gt;</span><br><span class="line">	&lt;&#x2F;property&gt;</span><br><span class="line">&lt;&#x2F;configuration&gt;</span><br></pre></td></tr></table></figure>
<h4 id="slaves"><a href="#slaves" class="headerlink" title="slaves"></a>slaves</h4><p>&nbsp;&nbsp;slaves指定子节点(DataNode)位置,因为yarn与HDFS分开启动,所以在yarn01中slaves指定的是NodeManager的位置。</p>
<h3 id="启动HA集群"><a href="#启动HA集群" class="headerlink" title="启动HA集群"></a>启动HA集群</h3><ol>
<li><p>启动zookeeper集群</p>
</li>
<li><p>启动JournalNode,一旦JNs启动，必须进行一次初始化同步在两个HA的NameNode，主要是为了元数据。<br>sbin/hadoop-daemon.sh start journalnode</p>
</li>
<li><p>格式化HDFS。<br>在namenode01上执行命令 hdfs namenode -format<br>格式化后会在根据core-site.xml中的hadoop.tmp.dir配置生成个文件,为了同步元数据,需要将tmp文件夹copy到namenode02上。<br>scp -r tmp/ namenode02:/home/application/hadoop-2.6.0/<br>也可以使用命令 hdfs namenode -bootstrapStandby</p>
</li>
<li><p>格式化ZKFC<br>在namenode01上执行命令 hdfs zkfc -formatZK</p>
</li>
<li><p>启动HDFS<br>在namenode01上执行命令 sbin/start-dfs.sh</p>
</li>
<li><p>启动Yarn<br>在yarn01上执行命令 sbin/start-yarn.sh。</p>
</li>
<li><p>因为自带的start-yarn.sh脚本并不会远程启动第二个RM,所以需要在yarn02上单独启动一个RM。<br>在yarn02上执行命令 sbin/yarn-daemon.sh start resourcemanager</p>
</li>
</ol>
<h3 id="管理命令"><a href="#管理命令" class="headerlink" title="管理命令"></a>管理命令</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">Usage: DFSHAAdmin [-ns &lt;nameserviceId&gt;]</span><br><span class="line">    [-transitionToActive &lt;serviceId&gt;]</span><br><span class="line">    [-transitionToStandby &lt;serviceId&gt;]</span><br><span class="line">    [-failover [--forcefence] [--forceactive] &lt;serviceId&gt; &lt;serviceId&gt;]</span><br><span class="line">    [-getServiceState &lt;serviceId&gt;]</span><br><span class="line">    [-checkHealth &lt;serviceId&gt;]</span><br><span class="line">    [-help &lt;command&gt;]</span><br></pre></td></tr></table></figure>
<p>&nbsp;&nbsp;描述了常用的命令，每个子命令的详细信息你应该运行”hdfs haadmin -help <command>“.</p>
<p><strong>transitionToActive &amp;&amp; transitionToStandby</strong> </p>
<p>&nbsp;&nbsp;切换NameNode的状态（Active或者Standby),这些子命令会使NameNode分别转换状态。</p>
<p><strong>failover</strong></p>
<p>&nbsp;&nbsp;启动两个NameNode之间的故障迁移。</p>
<p>&nbsp;&nbsp;这个子命令会从第一个NameNode迁移到第二个，如果第一个NameNode处于备用状态,这个命令只是没有错误的转换第二个节点到活动状态。如果第一个NameNode处于活跃状态,试图将优雅地转换到备用状态。如果失败,过滤方法(如由dfs.ha.fencing.methods配置)将尝试过滤直到成功。只有在这个过程之后第二个NameNode会转换为活动状态，如果没有过滤方法成功，第二个nameNode将不会活动并返回一个错误。</p>
<p><strong>getServiceState</strong></p>
<p>&nbsp;&nbsp;连接到NameNode，去判断现在的状态打印“standby”或者“active”去标准的输出。这个子命令可以被corn jobs或者是监控脚本使用，为了针对不同状态的NameNode采用不同的行为。</p>
<p><strong>checkHealth</strong></p>
<p>&nbsp;&nbsp;连接NameNode检查健康，NameNode能够执行一些诊断,包括检查如果内部服务正在运行。如果返回0表明NameNode健康，否则返回非0.可以使用此命令用于监测目的。</p>
<p>&nbsp;&nbsp;注意：这个功能实现的不完整，目前除了NameNode完全的关闭，其他全部返回成功。</p>

    </div>

    
    
    

    <footer class="post-footer">
          <div class="post-tags">
              <a href="/yuwanzi.io/tags/Hadoop/" rel="tag"># Hadoop</a>
              <a href="/yuwanzi.io/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/" rel="tag"># 大数据</a>
          </div>

        

          <div class="post-nav">
            <div class="post-nav-item">
                <a href="/yuwanzi.io/2016/07/14/2016-07-14-Hadoop02-MapReduce/" rel="prev" title="Hadoop学习笔记(2)-Mapreduce">
                  <i class="fa fa-chevron-left"></i> Hadoop学习笔记(2)-Mapreduce
                </a>
            </div>
            <div class="post-nav-item">
                <a href="/yuwanzi.io/2016/07/17/2016-07-17-Hadoop04-HBase/" rel="next" title="Hadoop学习笔记(4)-HBase">
                  Hadoop学习笔记(4)-HBase <i class="fa fa-chevron-right"></i>
                </a>
            </div>
          </div>
    </footer>
  </article>
</div>







<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      const activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      const commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>
</div>
  </main>

  <footer class="footer">
    <div class="footer-inner">


<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">玉丸子</span>
</div>
  <div class="powered-by">Erstellt mit  <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.js.org/muse/" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a>
  </div>

    </div>
  </footer>

  
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
<script src="/yuwanzi.io/js/utils.js"></script><script src="/yuwanzi.io/js/motion.js"></script><script src="/yuwanzi.io/js/schemes/muse.js"></script><script src="/yuwanzi.io/js/next-boot.js"></script>

  






  





</body>
</html>
